package com.chance.cc.crawler.development.bootstrap.autohome;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;

import java.util.ArrayList;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @author lt
 * @version 1.0
 * @date 2020-11-23 13:49:45
 * @email okprog@sina.com
 */
public class AutoHomeStart {

    private static final String domain = "autohome";
    public static final String dailyKwSite = "series";
    public static final String bmwKwSite = "bmwKw";
    public static final String adhocKwSite = "adhocKw";

    public static final String IS_OUTPUT_KV_CONTENT = "is_output_kv_content";
    private static final String OrderType = "order_type";//回复排序 1  发布排序 2
    private static final String InfoType = "info_type";//文章1  视频2

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        //H5168QRFNIU3804D
        //5F6B3610BB719FAA
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {
        //设置https协议访问
        System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2,SSLv3");
        //关键词 source record
        CrawlerRequestRecord keywordCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("autohome_keywords",turnPageItem)
                .httpUrl("http://192.168.1.217:9599//v1/meta/"+domain+"/keys?site=" + adhocKwSite)
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        //user agents source record
        CrawlerRequestRecord userAgentsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("taobao_user_agents", turnPageItem)
                .httpUrl("https://fake-useragent.herokuapp.com/browsers/0.1.11")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        //视频采集
       // CrawlerRequestRecord videoCrawler = doVideoCrawler();
        //文章采集
        CrawlerRequestRecord articleCrawler = doArticleCrawler();
        //论坛-最新发布采集
        CrawlerRequestRecord forumReleaseCrawler = doForumReleaseCrawler();
        //论坛-最新回复采集
        CrawlerRequestRecord forumReplyCrawler = doForumReplyCrawler();
        //论坛-采集
        CrawlerRequestRecord forumCrawler = doForumCrawler();
        //车家号-最新文章采集
        CrawlerRequestRecord cheArticleCrawler = doCheArticleCrawler();
        //车家号-最新视频采集
        CrawlerRequestRecord cheVideoCrawler = doCheVideoCrawler();
        //口碑采集
        CrawlerRequestRecord praiseCrawler = doPraiseCrawler();
        //经销商发文采集
        CrawlerRequestRecord dealerCrawler = doDealerArticleCrawler();

        CrawlerRequestRecord forumOneCrawler = doForumOneCrawler();


        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain, domain + "_trigger", System.currentTimeMillis(), domain + "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
//                .consoleResultPipeline("console") //控制台输t出
//                .fileResultPipeline("D:\\chance\\data\\autohome\\autohome_forum.json", false)
//                .fileResultPipeline("file","D:\\chance\\data\\autohome\\autohome_che_article.json", false)
//                .fileResultPipeline("file_redis","D:\\chance\\data\\autohome\\autohome_urls.json", false)
//                .requestRecord(articleCrawler)  //more job
              //  .requestRecord(videoCrawler)
//                .requestRecord(forumReleaseCrawler)
//                .requestRecord(forumReplyCrawler)
//                .requestRecord(forumCrawler)
//                .requestRecord(forumOneCrawler)
                .requestRecord(cheArticleCrawler)
                //.requestRecord(cheVideoCrawler)
//                .requestRecord(praiseCrawler)
//                .requestRecord(doArticleUrls())
//                .requestRecord(dealerCrawler)
                .supportRecord(keywordCrawlerRecord)
//                .supportRecord(userAgentsCrawlerRecord)
                .crawlerThreadNum(8)
                .build("com.chance.cc.crawler.development.scripts.autohome");
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag(IS_OUTPUT_KV_CONTENT,true);
        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domain,"comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 30,null));
        cheArticleCrawler.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        devCrawlerController.start();
    }
    public static final String indexUrl = "https://www.autohome.com.cn/";

    public static CrawlerRequestRecord doArticleUrls(){
        String site = "article_urls";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPage)
                .domain(domain)
                .httpUrl(indexUrl + "#" +site)
                .recordKey(indexUrl + "#" +site)
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.key)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .needParsed(true)
                .needWashed(false)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addCustomKV("filePath","D:\\py\\通用宝马\\测试\\urls.json");
        return requestRecord;
    }

    public static CrawlerRequestRecord doArticleCrawler(){
        String site = "article";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://www.autohome.com.cn/channel2/union/list.html")
                .httpUrl("https://www.autohome.com.cn/channel2/union/list.html")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
//                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
//                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
//                .needWashed(false)
//                .needParsed(false)
                .build();
//        requestRecord.setDownload(false);
//        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doVideoCrawler(){
        String site = "video";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://v.autohome.com.cn/")
                .httpUrl("https://v.autohome.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
//                .needParsed(false)
//                .needWashed(false)
                .build();
//        requestRecord.setDownload(false);
//        requestRecord.setSkipPipeline(true);

//        requestRecord.tagsCreator().bizTags().addCustomKV("crawler_site","video");
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doForumReleaseCrawler(){
        String site = "forum_release";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://www.autohome.com.cn/")
                .httpUrl("https://www.autohome.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(1,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doForumReplyCrawler(){
        String site = "forum_reply";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://www.autohome.com.cn/")
                .httpUrl("https://www.autohome.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*1,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doForumCrawler(){
        String site = "forum";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("http://club.autohome.com.cn/bbs/thread/cc67704fac031a50/97029406-1.html")
                .httpUrl("http://club.autohome.com.cn/bbs/thread/cc67704fac031a50/97029406-1.html")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 30,null))
                .proxy(proxy)
//                .needParsed(false)
                .needWashed(true)
                .build();
//        requestRecord.setDownload(false);
//        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(site);
        requestRecord.tagsCreator().bizTags().addCustomKV(OrderType,"1");
        return requestRecord;
    }

    public static CrawlerRequestRecord doForumOneCrawler(){
        String site = "forum";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://www.autohome.com.cn/770/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.key)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 17,null))
                .proxy(proxy)
//                .needParsed(false)
//                .needWashed(false)
                .build();
//        requestRecord.setDownload(false);
//        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(site);
        requestRecord.tagsCreator().bizTags().addCustomKV(OrderType,"1");
        return requestRecord;
    }


    public static CrawlerRequestRecord doCheArticleCrawler(){
        String site = "che_article";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://chejiahao.autohome.com.cn/?infotype=1")
                .httpUrl("https://chejiahao.autohome.com.cn/?infotype=1")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(site);
        requestRecord.tagsCreator().bizTags().addCustomKV(InfoType,"1");
        return requestRecord;
    }

    public static CrawlerRequestRecord doCheVideoCrawler(){
        String site = "che_video";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://chejiahao.autohome.com.cn/?infotype=3")
                .httpUrl("https://chejiahao.autohome.com.cn/?infotype=3")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(1,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doPraiseCrawler(){
        String site = "praise";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://www.autohome.com.cn/")
                .httpUrl("https://www.autohome.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*42,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doDealerArticleCrawler(){
        String site = "dealer";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://dealer.autohome.com.cn/all/")
                .httpUrl("https://dealer.autohome.com.cn/all/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    @Test
    public void testDealer(){
//        HttpConfig httpConfig = HttpConfig.me("test");
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(false);
//
//        HttpClientDownloader downloader = new HttpClientDownloader();
//        downloader.setClientGenerator(new HttpClientGenerator());
//        downloader.setProxyProvider(new SiteConfigProxyProvider());
//
//
//
//        String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?jsv=2.6.1&appKey=12574478&t=1623325984223&sign=18a82f578012a6e9dffb235089cd9596&api=mtop.taobao.detail.getdetail&v=6.0&isSec=0&ecode=0&AntiFlood=true&AntiCreep=true&ttid=2018%40taobao_h5_9.9.9&H5Request=true&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22id%22%3A%22623337888932%22%2C%22hybrid%22%3A%22true%22%2C%22ttid%22%3A%221608030065155%40taobao_android_10.0.0%22%2C%22itemNumId%22%3A%22623337888932%22%2C%22itemId%22%3A%22623337888932%22%2C%22exParams%22%3A%22%7B%5C%22id%5C%22%3A%5C%22623337888932%5C%22%2C%5C%22hybrid%5C%22%3A%5C%22true%5C%22%2C%5C%22ttid%5C%22%3A%5C%221608030065155%40taobao_android_10.0.0%5C%22%7D%22%2C%22detail_v%22%3A%228.0.0%22%2C%22utdid%22%3A%221%22%7D";
//        HttpRequest httpRequest = new HttpRequest();
//        httpRequest.setUrl(url);
//        httpRequest.addHeader("Cookie","cna=krlIGXvjR0ECATohFCJGKHyc");
//        httpRequest.addHeader("Host","h5api.m.taobao.com");
//        httpRequest.addHeader("User-Agent","Mozilla/5.0 (Linux; U; Android 8.0.0; zh-CN; MI 5s Build/OPR1.170623.032) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/69.0.3497.100 UWS/3.22.1.130 Mobile Safari/537.36 AliApp(TB/10.0.0) UCBS/2.11.1.1 TTID/1608030065155@taobao_android_10.0.0 WindVane/8.5.0 1080X1920 UT4Aplus/0.2.16");
//        httpRequest.addHeader("Referer","https://h5.m.taobao.com/");
//
//        HttpPage httpPage = downloader.download(httpRequest, httpConfig);
//        System.out.println(httpPage.getRawText());
//        List<Map> maps = new ArrayList<>();
//        List<Integer> list = Arrays.asList(1, 2, 3, 4);
//        Map map = null;
//        for (Integer a : list) {
//            if (list.indexOf(a) == 0 || list.indexOf(a) % 4 == 0){
//                map = new HashMap<>(4);
//                maps.add(map);
//            }
//            map.put(a,"s");
//        }
//        list.forEach(a -> {
//            if (list.indexOf(a) == 0 || list.indexOf(a) % 4 == 0){
//                map = new HashMap<>(4);
//                maps.add(map);
//            }
//            map.put(a,"s");
//        });
//        System.out.println(maps);

        ArrayList list = new ArrayList();
        list.add(1);
        list.add(2);

    }

    static class TestSearchNews implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestSearchNews(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?jsv=2.6.1&appKey=12574478&t=1623324046639&sign=5fe4ee7b4e6b7f178bb964f430e11764&api=mtop.taobao.detail.getdetail&v=6.0&isSec=0&ecode=0&AntiFlood=true&AntiCreep=true&ttid=2018%40taobao_h5_9.9.9&H5Request=true&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22id%22%3A%22623337888932%22%2C%22hybrid%22%3A%22true%22%2C%22ttid%22%3A%221608030065155%40taobao_android_10.0.0%22%2C%22itemNumId%22%3A%22623337888932%22%2C%22itemId%22%3A%22623337888932%22%2C%22exParams%22%3A%22%7B%5C%22id%5C%22%3A%5C%22623337888932%5C%22%2C%5C%22hybrid%5C%22%3A%5C%22true%5C%22%2C%5C%22ttid%5C%22%3A%5C%221608030065155%40taobao_android_10.0.0%5C%22%7D%22%2C%22detail_v%22%3A%228.0.0%22%2C%22utdid%22%3A%221%22%7D";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(url);
            httpRequest.addHeader("Cookie","cna=krlIGXvjR0ECATohFCJGKHyc");
            httpRequest.addHeader("Host","h5api.m.taobao.com");
            httpRequest.addHeader("User-Agent","Mozilla/5.0 (Linux; U; Android 8.0.0; zh-CN; MI 5s Build/OPR1.170623.032) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/69.0.3497.100 UWS/3.22.1.130 Mobile Safari/537.36 AliApp(TB/10.0.0) UCBS/2.11.1.1 TTID/1608030065155@taobao_android_10.0.0 WindVane/8.5.0 1080X1920 UT4Aplus/0.2.16");
            httpRequest.addHeader("Referer","https://h5.m.taobao.com/");

            HttpPage httpPage = downloader.download(httpRequest, httpConfig);
            System.out.println(httpPage.getRawText());
            JSONObject jsonObject = JSONObject.parseObject(httpPage.getRawText());
            JSONObject resultObj = jsonObject.getJSONArray("result").getJSONObject(0);
            int commentCount = resultObj.getIntValue("commentCount");
            System.out.println(commentCount);
//            for(int i=1;i<2;i++){
//                httpRequest.setUrl(url);
//                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
////                System.out.println(httpPage.getRawText());
//                Html responseHtml = httpPage.getHtml();
//                List<Selectable> nodes = responseHtml.xpath("//ul[@class=\"filter-box\"]/li[@class=\"row row-hide data-brand-item\"]/div/a[@class=\"item\"]").nodes();
//                for (Selectable node : nodes) {
//
//                    String articleUrl = node.xpath("./@href").get();
//                    String releaseTime = node.xpath("./text()").all().toString();
//
//
//                    System.out.println(articleUrl);
//                    System.out.println(releaseTime);
//                }
//
//            }
        }
    }
}
